# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import logging
import base64
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
import random,time
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from scrapy.http import HtmlResponse

# 随机请求头类
class UserAgentDownloadMiddleware(object):
    #  随机请求头中间件
    USER_AGENT = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]

    def process_request(self,request,spider):
        user_agent = random.choice(self.USER_AGENT)
        request.headers['User-Agent'] = user_agent

# selenium
class LoadDataMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def spider_opened(self,spider):
        # 创建Selenium的Chrome浏览器对象
        self.driver_path = r"D:\Chrome\chromedriver.exe"  # 驱动

        self.options = Options()
        self.options.add_argument('--headless') # 浏览器不提供可视化页面
        self.options.add_argument('--start-maximized')  # 最大化运行（全屏窗口）,不设置，取元素会报错
        self.options.add_argument('--disable-gpu')  # 禁用GPU加速
        self.chrome = Chrome(executable_path=self.driver_path,options=self.options)   # 路径

    def process_request(self,request, spider):
        # 针对一药网的详情页进行selenium采集数据
        if spider.name == "onemedicine":
            if request.url in spider.detail_list:
                self.chrome.get(request.url)

                time.sleep(0.3)
                html = self.chrome.page_source  # 获取源码
                # print(html)
                new_response = HtmlResponse(request.url, body=html.encode('gbk','ignore'))
                return new_response
            # else:
            #     return response


    def spider_closed(self,spider):
        self.chrome.quit()

# 代理ip
logger = logging.getLogger(__name__)
# 隧道id和密码
tid = "t10611534979779"
password = "0hvsjx04"
# 隧道host和端口
tunnel_master_host = "tps161.kdlapi.com"
tunnel_master_port = "15818"
# 备用隧道host和端口
tunnel_slave_host = "tps163.kdlapi.com"
tunnel_slave_port = "15818"
# 切换阀值
threshold = 3

# 代理中间件
class ProxyDownloadMiddleware(object):

    def process_request(self, request, spider):
        global threshold
        if threshold > 0:
            host, port = tunnel_master_host, tunnel_master_port
        else:
            host, port = tunnel_slave_host, tunnel_slave_port
        if request.url.startswith("http://"):
            proxy_url = 'http://{host}:{port}'.format(host=host, port=port)
        elif request.url.startswith("https://"):
            proxy_url = 'https://{host}:{port}'.format(host=host, port=port)
        request.meta['proxy'] = proxy_url  # 设置代理
        logger.debug("using proxy: {}".format(request.meta['proxy']))
        # 隧道代理需要进行身份验证
        #
        # 用户名和密码需要先进行base64编码，然后再赋值
        username_password = "{tid}:{password}".format(tid=tid, password=password)
        b64_username_password = base64.b64encode(username_password.encode('utf-8'))
        request.headers['Proxy-Authorization'] = 'Basic ' + b64_username_password.decode('utf-8')
        threshold -= 1
        return None